from utils.simhash import hamming_dist
import struct
import base64

BIT_THRESHOLD = 3

NUM_BITS = 64


def decode_simhash(sh):
    b64 = base64.b64decode(sh)
    return struct.unpack('Q', b64)


def get_dup_index(dedup_hashes, h, key, text_len, num_bits=NUM_BITS, bit_threshold=BIT_THRESHOLD):
    return [k for k in range(len(dedup_hashes)) if
            hamming_dist(dedup_hashes[k], h, num_bits) < bit_threshold or (key == 'text_simhash' and text_len < 10)]


def article_deduplicate(data, num_bits=NUM_BITS, bit_threshold=BIT_THRESHOLD, key='title_simhash', remove=True):
    hashes = []
    for d in data:
        simhash = d.get(key, '')
        h = decode_simhash(simhash)
        hashes.append(h[0])

    dedup_hashes = []

    out = []
    k = 0
    for i in range(len(data)):
        d = data[i]
        if remove:
            del (d[key])
        if i == 0:
            dedup_hashes.append(hashes[i])
            out.append(d)
            k += 1
        else:
            indices = get_dup_index(dedup_hashes, hashes[i], key, d.get('text_len', 0), num_bits, bit_threshold)
            if indices:
                if not remove:
                    out[indices[0]]['duplicates'] = out[indices[0]].get('duplicates', []) + [d]
                out[indices[0]]['duplicated_ids'] = out[indices[0]].get('duplicated_ids', []) + [d.get('article_id')]
            else:
                dedup_hashes.append(hashes[i])
                out.append(d)
                k += 1

    return out
